{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:06:37.939545900Z",
     "start_time": "2024-07-19T08:06:32.968012Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.9.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.2\n",
      "sklearn 1.5.0\n",
      "torch 2.3.1+cpu\n",
      "cpu\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "\n",
    "实际上我们从chapter_2 开始就在使用该模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:12:39.468433200Z",
     "start_time": "2024-07-19T08:12:39.460169400Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "(1000,)"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "class RandomDataset(Dataset):\n",
    "    def __init__(self, labels):\n",
    "        self.labels = labels\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        data = torch.mul(torch.randn(2), 0.1) + self.labels[index] # 生成两个服从标准正态分布的随机数（两个维度），然后乘以0.1，再加上label\n",
    "        label = self.labels[index]\n",
    "        return data, label # 返回的是一个元组,这里返回的数据格式决定我们遍历这个数据集时，每次迭代返回的数据格式\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "#np.random.randint(2, size=1000)生成了一个长度为1000的随机标签数组，每个元素都是0或1\n",
    "rd = RandomDataset(np.random.randint(2, size=1000))\n",
    "print(\"数据集的长度：\", len(rd))\n",
    "\n",
    "# 可视化随机数据集\n",
    "xx = []\n",
    "yy = []\n",
    "labels = []\n",
    "\n",
    "for (x, y), label in rd:\n",
    "    xx.append(x)\n",
    "    yy.append(y)\n",
    "    labels.append(label)\n",
    "\n",
    "    \n",
    "plt.scatter(xx, yy, s=10, c=labels) # s是点的大小，c是颜色\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:11:18.763467Z",
     "start_time": "2024-07-19T08:11:18.753493Z"
    }
   },
   "source": [
    "## DataLoader"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([0.9230, 0.8695])"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-25T08:20:52.739770600Z",
     "start_time": "2024-04-25T08:20:52.140694500Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0453, -0.0975],\n",
      "        [ 0.1502, -0.0411],\n",
      "        [ 0.0483, -0.2111],\n",
      "        [ 0.7816,  1.0501],\n",
      "        [ 0.0982,  0.0043],\n",
      "        [ 1.0772,  0.8873],\n",
      "        [ 0.9693,  1.2358],\n",
      "        [-0.1285,  0.0805]])\n",
      "tensor([0, 0, 0, 1, 0, 1, 1, 0], dtype=torch.int32)\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "ld = DataLoader(rd, batch_size=8, shuffle=True) # batch_size是每次迭代返回的数据个数，shuffle是是否打乱数据集\n",
    "\n",
    "for data, label in ld:\n",
    "    print(data)\n",
    "    print(label)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tfrecord\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-19T08:13:01.285506800Z",
     "start_time": "2024-07-19T08:13:01.277636200Z"
    }
   },
   "source": [
    "复现论文的时候更多地发现使用 pyarrow 而不是 tfrecord\n",
    "\n",
    "如果想在pytorch里使用tfrecord，可以参考 https://github.com/vahidk/tfrecord"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 2])\n",
      "torch.Size([8])\n"
     ]
    }
   ],
   "execution_count": 9
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
